package textstat

import collection.mutable.HashMap
import io.Source


/**
 * Created by IntelliJ IDEA.
 * User: user
 * Date: May 7, 2010
 * Time: 3:43:09 PM
 * To change this template use File | Settings | File Templates.
 */

object LiftOverDecomposition {
  val probab = new HashMap[String,Double]
  val lifts = new HashMap[String,Double]

  def main(args:Array[String]){
    val counts = new HashMap[String,Int]

    def txt = getClass.getResourceAsStream("/textstat/Book1.html")
    val sourceLines = Source.fromInputStream(txt).getLines()
    val sb = new StringBuilder
    sourceLines.foreach(s=>sb.append(s+" "))
    val src = sb.toString.toLowerCase
    for(len <- 1 to 10){
      for( i<- 0 to (src.length - len)){
        val key = src.substring(i, i+len)
        counts.update(key, counts.getOrElse(key,0)+1)
      }
    }

    //val probab = counts.map{ case (s,i) => (s,i.toDouble/(src.length+1-s.length)) }
    println("counts done")
    counts.foreach{ case (s,i) => if(i>10) probab.update(s, i.toDouble/(src.length+1-s.length) ) }
    println("probab done")
    probab.foreach{ case (s,i) => if (s.length>=2) lifts.update(s, lift(s)) }
    println("lifts done")

    val maxLifts = lifts.toList.sortBy(-_._2)
    println("sorts done")

    maxLifts.slice(0,500).foreach{
      case (s,l) => println(s+" "+counts.get(s).get+" "+l)
    }

  //  val chars = src.toList
  //  for( c <- chars ){
  //    val key = ""+c.toLower
  //    counts.update(key, counts.getOrElse(key,0)+1)
  //  }
  //
  //  val countsSorted = counts.toList.sortWith( (a,b) => a._2 > b._2 )
  //  countsSorted.foreach( a => println(a) )

    System.out.println( lift("знайка") )
    System.out.println( lift(" знайка") )
    System.out.println( lift("знайка ") )
    System.out.println( lift("ка") )


  }

  def lift(s:String):Double={
    val plist = for{i<-0 until s.length-1
       (s1,s2) = s.splitAt(i+1)
      p1 = probab.get(s1).get
      p2 = probab.get(s2).get
    }
      yield p1*p2

    val ps = probab.get(s).get
    ps / plist.max
  }
}